/**
 * 文件名:BaiDuBaiKeCrawler.java
 * 包名:com.wx.demo.wcd.crawler
 * 创建时间:2017年2月23日上午11:49:03
 * <p>Copyright ®医界互联 TUS 平台版权所有。</p>
 *
 */

package com.wx.demo.wcd.crawler;

import java.util.HashMap;
import java.util.Map;

import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;

import com.wx.demo.wcd.domain.po.AuthorPO;
import com.wx.demo.wcd.domain.qo.AuthorQO;
import com.wx.demo.wcd.service.AuthorService;
import com.wx.demo.wcd.service.impl.AuthorServiceImpl;

/**
 * 类名:BaiDuBaiKeCrawler <br/>
 * 创建时间: 2017年2月23日 上午11:49:03 <br/>
 * 〈功能简述〉百度百科爬取诗人词人属性数据. <br/>
 * 〈功能详细描述〉.<br/>
 * 
 * @author 王笑 Email:wangxiao@yijiehulian.com.
 * @version
 * @since JDK 1.8
 * @see
 */
public class BaiDuBaiKeCrawler extends BreadthCrawler {

	private Logger logger = Logger.getLogger(getClass());

	/**
	 * 读取Spring配置信息
	 */
	static AbstractApplicationContext ctx = new ClassPathXmlApplicationContext(
			new String[] { "classpath*:/spring/spring-dao.xml",
					"classpath*:/spring/spring-mvc.xml",
					"classpath*:/spring/spring-service.xml" });
	static AuthorService authorService = (AuthorServiceImpl) ctx
			.getBean("authorService");

	public static void main(String[] args) throws Exception {
		BaiDuBaiKeCrawler crawler = new BaiDuBaiKeCrawler("depth_crawler", true);
		AuthorQO aq = new AuthorQO();
		aq.setDescription_notnull("NF");// 查询人物描述为null的
		aq.addOrderBy("dynasty", true);// 根据人物的朝代分组
		aq.setPageSize(-100);// 查询所有
		java.util.List<AuthorPO> las = authorService.selectCondition1(aq);

		if (las.size() > 0) {
			for (AuthorPO ap : las) {
				String name = ap.getAuthorName();
				if (name != null && !"".equals(name)) {
					crawler.addSeed(new CrawlDatum("http://www.baidu.com/s?wd="
							+ name).meta("depth", "1"));
				}
			}
		}

		/*
		 * crawler.addSeed(new CrawlDatum("http://www.baidu.com/s?wd=" + "何希尧")
		 * .meta("depth", "1"));
		 */

		/* 不要爬取包含"#"的链接 */
		crawler.addRegex("-.*#.*");
		/* 设置执行间隔 */
		crawler.setExecuteInterval(3000L);
		/* 线程数 */
		crawler.setThreads(50);
		/* 设置每次迭代中爬取数量的上限 */
		crawler.setTopN(las.size());
		// crawler.setTopN(4);
		// 爬取深度
		crawler.start(5);
	}

	public BaiDuBaiKeCrawler(String crawlPath, boolean autoParse) {
		super(crawlPath, autoParse);
	}

	@Override
	public void visit(Page page, CrawlDatums next) {
		String name1 = "";
		String name2 = "";
		try {
			String url = page.getUrl();// 当前url
			// 百度搜索
			if (url.matches("http://www.baidu.com/s\\?wd=(.*)")) {

				// 互动百科根节点
				Elements wkRt = page.select("div[id=content_left]")
						.select("div[class=f13]").select("a[class=c-showurl]")
						.not("b");

				// 百度百科根节点
				Elements bkRt = page.select("div[id=content_left]")
						.select("div[class=result-op c-container xpath-log]")
						.select("div[class=c-row]")
						.select("span[class=c-showurl]");

				// 互动百科
				for (Element emt : wkRt) {
					String wk = emt.text();
					if (wk != null && !"".equals(wk)) {
						if (wk.matches("www.baike.com/wiki/(.*)")) {
							// 互动百科父节点
							Elements wkPt = emt.parent().parent()
									.select("h3[class=t]")
									.select("a[target=_blank]");
							// 互动百科节点链接
							String wkHf = wkPt.attr("href");
							if (wkHf != null && !"".equals(wkHf)) {
								next.add(wkHf);
							}
						}
					}
				}

				// 百度百科
				for (Element emt : bkRt) {
					String wk = emt.text();
					if (wk != null && !"".equals(wk)) {
						if (wk.matches("baike.baidu.com/(.*)")) {
							// 百度百科父节点
							Elements bkPt = emt.parent().parent().parent()
									.parent()
									.select("h3[class=t c-gap-bottom-small]")
									.select("a[target=_blank]");
							// 百度百科节点链接
							String bkHf = bkPt.attr("href");
							if (bkHf != null && !"".equals(bkHf)) {
								next.add(bkHf);
							}
						}
					}
				}
			}
			// 百度百科
			if (url.matches("http://www.baidu.com/link\\?url=(.*)")) {

				// 互动百科根节点
				Elements wkRt = page.select("div[class=l w-640]");
				// 互动百科人物属性名称
				Elements wkDt = wkRt.select("div[id=datamodule]").select("td")
						.select("strong");
				// 互动百科人物属性值
				Elements wkDd = wkRt.select("div[id=datamodule]").select("td")
						.select("span");
				// 互动百科人物名
				Elements wkName = wkRt.select("div[class=content-h1]").select(
						"h1");
				name1 = wkName.text().trim();
				if (name1 != null && !"".equals(name1)) {
					if (name1.indexOf("[") > 0) {
						name1 = name1.substring(0, name1.indexOf("["));
					}
				}
				AuthorPO ap1 = new AuthorPO();
				Map<String, String> mp1 = new HashMap<String, String>();
				for (int i = 0; i < wkDt.size(); i++) {
					String key = wkDt.get(i).text().trim();
					key = key.substring(0, key.indexOf("："));
					String value = wkDd.get(i).text().trim();
					mp1.put(key, value);
				}
				ap1.setDescription(mp1.toString());
				if ((name1 != null && !"".equals(name1))
						&& (ap1.getDescription() != null && !"".equals(ap1
								.getDescription()))) {
					AuthorQO aq = new AuthorQO();
					aq.setAuthorName(name1);// 作者姓名
					aq.setDescription_notnull("NF");// 查询人物描述为null的
					java.util.List<AuthorPO> las = authorService
							.selectCondition1(aq);
					if (las.size() > 0) {
						logger.info(las.get(0).inputDescription());
						AuthorQO aq1 = new AuthorQO();
						aq1.setAuthorId(las.get(0).getAuthorId());
						int rows = authorService.edit(ap1, aq1);
						if (rows > 0) {
							logger.info("数据抓取并存储成功！");
						} else {
							logger.info("数据抓取并存储失败！");
						}
					}
				}

				// 百度百科根节点
				Elements bkRt = page.select("div[class=main-content]");
				// 百度百科人物属性名称
				Elements bkDt = bkRt.select("dt[class=basicInfo-item name]");
				// 百度百科人物属性值
				Elements bkDd = bkRt.select("dd[class=basicInfo-item value]");
				// 百度百科人物名
				Elements bkName = bkRt.select(
						"dl[class=lemmaWgt-lemmaTitle lemmaWgt-lemmaTitle-]")
						.select("h1");

				name2 = bkName.text().trim();
				if (name2 != null && !"".equals(name2)) {
					if (name2.indexOf("[") > 0) {
						name2 = name2.substring(0, name2.indexOf("["));
					}
				}

				AuthorPO ap2 = new AuthorPO();
				Map<String, String> mp2 = new HashMap<String, String>();
				for (int i = 0; i < bkDt.size(); i++) {
					String key = bkDt.get(i).text().trim().replace(" ", "");
					String value = bkDd.get(i).text().trim();
					mp2.put(key, value);
				}
				ap2.setDescription(mp1.toString());
				if ((name2 != null && !"".equals(name2))
						&& (ap2.getDescription() != null && !"".equals(ap2
								.getDescription()))) {
					AuthorQO aq = new AuthorQO();
					aq.setAuthorName(name2);// 作者姓名
					aq.setDescription_notnull("NF");// 查询人物描述为null的
					java.util.List<AuthorPO> las = authorService
							.selectCondition1(aq);
					if (las.size() > 0) {
						logger.info(las.get(0).inputDescription());
						AuthorQO aq1 = new AuthorQO();
						aq1.setAuthorId(las.get(0).getAuthorId());
						int rows = authorService.edit(ap2, aq1);
						if (rows > 0) {
							logger.info("数据抓取并存储成功！");
						} else {
							logger.info("数据抓取并存储失败！");
						}
					}
				}
			}
		} catch (Exception e) {
			logger.debug(e.getMessage());
			logger.info(e.getMessage());
		}

	}

	/**
	 * 
	 * fmtDynasty:年代格式化. <br/>
	 * 
	 * @author 王笑 Email:wangxiao@yijiehulian.com.;
	 * @param dynasty
	 * @return
	 * @since JDK 1.8
	 */
	public static String fmtDynasty(String dynasty) {
		if ("02".equals(dynasty)) {
			return "先秦";
		} else if ("03".equals(dynasty)) {
			return "两汉";
		} else if ("04".equals(dynasty)) {
			return "魏晋";
		} else if ("05".equals(dynasty)) {
			return "南北朝";
		} else if ("06".equals(dynasty)) {
			return "隋代";
		} else if ("07".equals(dynasty)) {
			return "唐代";
		} else if ("08".equals(dynasty)) {
			return "五代";
		} else if ("09".equals(dynasty)) {
			return "宋代";
		} else if ("10".equals(dynasty)) {
			return "金朝";
		} else if ("11".equals(dynasty)) {
			return "元代";
		} else if ("12".equals(dynasty)) {
			return "明代";
		} else if ("13".equals(dynasty)) {
			return "清代";
		} else if ("14".equals(dynasty)) {
			return "近代";
		} else {
			return "";
		}
	}
}
